#!/bin/sh

. "$SPEC_PATH/abstract/dbops-major-version-upgrade"

e2e_test_extra_hash() {
  "$SHELL" "$PROJECT_PATH/stackgres-k8s/ci/build/build-functions.sh" path_hash \
    "$(realpath --relative-to "$PROJECT_PATH" "$SPEC_PATH/abstract/dbops-major-version-upgrade")"
}

e2e_test_install() {
  kubectl create ns "$CLUSTER_NAMESPACE"

  echo "Creating SGCluster with callbacks"

  kubectl create secret -n "$CLUSTER_NAMESPACE" generic credentials --from-literal=superuser-password=sup3rus3r
  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "2" \
    --set-string 'cluster.postgres.version='"$E2E_MAJOR_SOURCE_POSTGRES_VERSION" \
    --set-string 'cluster.postgres.extensions[0].name=pg_cron' \
    --set-string 'configurations.postgresconfig.postgresql\.conf.shared_preload_libraries=pg_stat_statements\,auto_explain\,pg_cron' \
    --set-string 'configurations.postgresconfig.postgresql\.conf.cron\.host=/var/run/postgresql' \
    --set-string 'cluster.configurations.credentials.users.superuser.password.name=credentials' \
    --set-string 'cluster.configurations.credentials.users.superuser.password.key=superuser-password' \
    --set-string 'cluster.configurations.patroni.initialConfig.postgresql.callbacks.on_role_change=/callbacks/pre_promote' \
    --set-string 'cluster.configurations.patroni.initialConfig.postgresql.before_stop=/callbacks/before_stop' \
    --set-string 'cluster.pods.customVolumeMounts.patroni[0].name=custom-callbacks' \
    --set-string 'cluster.pods.customVolumeMounts.patroni[0].mountPath=/callbacks' \
    --set-string 'cluster.pods.customVolumes[0].name=callbacks' \
    --set-string 'cluster.pods.customVolumes[0].configMap.name=callbacks' \
    --set 'cluster.pods.customVolumes[0].configMap.defaultMode='"$(printf %d 0555)" \
    --set 'cluster.pods.customVolumes[0].configMap.optional=false'

  cat << EOF | kubectl create -n "$CLUSTER_NAMESPACE" -f -
---
apiVersion: v1
kind: Service
metadata:
  name: pgbouncer
spec:
  type: ClusterIP
  selector:
    app: pgbouncer
  ports:
  - name: pgbouncer
    port: 5432
    protocol: TCP
    targetPort: pgbouncer
---
apiVersion: v1
kind: Service
metadata:
  name: entrypoint
spec:
  type: ExternalName
  externalName: "$CLUSTER_NAME.$CLUSTER_NAMESPACE.svc.cluster.local"
---
apiVersion: v1
kind: Endpoints
metadata:
  name: entrypoint
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: pgbouncer
spec:
  selector:
    matchLabels:
      app: pgbouncer
  template:
    metadata:
      labels:
        app: pgbouncer
    spec:
      terminationGracePeriodSeconds: 0
      containers:
      - name: pgbench
        image: quay.io/ongres/postgres-util:v17.2-build-6.38
        command:
        - sh
        - -c
        - while true; do sleep 300; done;
      - name: pgbouncer
        image: quay.io/ongres/pgbouncer:v1.31.3-build-6.38
        command:
        - sh
        - /usr/local/bin/start-pgbouncer.sh
        ports:
        - containerPort: 5432
          name: pgbouncer
          protocol: TCP
        volumeMounts:
        - name: dynamic
          mountPath: /etc/pgbouncer
        - name: config
          mountPath: /etc/pgbouncer/pgbouncer.ini
          subPath: pgbouncer.ini
        - name: config
          mountPath: /usr/local/bin/start-pgbouncer.sh
          subPath: start-pgbouncer.sh
      volumes:
      - name: dynamic
        emptyDir: {}
      - name: config
        configMap:
          defaultMode: 0444
          name: pgbouncer
          optional: false
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: pgbouncer
data:
  pgbouncer.ini: |
    [databases]

    * = host=entrypoint port=5432

    [pgbouncer]
    listen_addr=0.0.0.0
    listen_port=5432

    pool_mode=transaction
    max_client_conn=1000
    default_pool_size=100
    max_db_connections=0
    max_user_connections=0

    auth_type=md5
    auth_file=/etc/pgbouncer/userlist.txt
    auth_user=postgres
    auth_query=SELECT usename, passwd FROM pg_shadow WHERE usename=$1

    admin_users=pgbouncer_admin
    stats_users=pgbouncer_stats
    application_name_add_host=1
    ignore_startup_parameters=extra_float_digits

    server_check_query=;
  start-pgbouncer.sh: |
    #!/bin/sh
    printf '"%s" "%s"\n' "postgres" "sup3rus3r" >> /etc/pgbouncer/userlist.txt
    printf '"%s" "%s"\n' "pgbouncer_admin" "pgb0unc3r" >> /etc/pgbouncer/userlist.txt
    printf '"%s" "%s"\n' "pgbouncer_stats" "pgb0unc3r" >> /etc/pgbouncer/userlist.txt
    exec pgbouncer /etc/pgbouncer/pgbouncer.ini
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: callbacks
data:
  pre_promote: |
    #!/bin/sh
    set -x
    if [ "$#" = 0 ] || [ "x$2" = xprimary ]
    then
      until psql -q -tA -c 'SELECT pg_is_in_recovery()' | grep -qxF f
      do
        true
      done
      psql postgresql://pgbouncer_admin:pgb0unc3r@pgbouncer/pgbouncer -c RESUME
      psql postgresql://pgbouncer_admin:pgb0unc3r@pgbouncer/pgbouncer -tA -c 'SHOW STATE' | grep -q 'paused|no'
    fi
  before_stop: |
    #!/bin/sh
    set -x
    PATRONI_NAME="$(cat /etc/hostname)"
    PATRONI_HISTORY="$(patronictl history -f tsv | tail -n +2)"
    PATRONI_LIST="$(patronictl list -f tsv | tail -n +2)"
    if {
        [ "x$PATRONI_HISTORY" = x ] \
        && ! printf %s "$PATRONI_LIST" | grep -v $'^[^\t]\+\t'"$PATRONI_NAME"$'\t' | grep -q $'^[^\t]\+\t[^\t]\+\t[^\t]\+\tLeader\t'
      } \
      || printf %s "$PATRONI_HISTORY" | grep -q $'^[^\t]\+\t[^\t]\+\t[^\t]\+\t[^\t]\+\t'"$PATRONI_NAME"'$'
    then
      psql postgresql://pgbouncer_admin:pgb0unc3r@pgbouncer/pgbouncer -c PAUSE
    fi
    exit 0
EOF

  wait_pods_running "$CLUSTER_NAMESPACE" 3
  wait_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
}

e2e_test_uninstall() {
  if [ "$E2E_CLEANUP_RUNBOOKS" = true ]
  then
    helm_cleanup_chart "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"

    k8s_async_cleanup_namespace "$CLUSTER_NAMESPACE"
  fi
}

e2e_test() {
  # run_test "Checking that is possible to have zerodowntime using PgBouncer in transaction mode" zerodowntime_check
  run_test "Checking that is possible to have a major version upgrade with zerodowntime using PgBouncer in transaction mode" zerodowntime_major_version_upgrade_check
}

zerodowntime_check() {
  kubectl exec -n "$CLUSTER_NAMESPACE" deploy/pgbouncer -c pgbench -- pgbench postgresql://postgres:sup3rus3r@localhost/postgres -i 2>&1 | tee "$LOG_PATH/bench"
  if ! grep -q '^command terminated with exit code [1-9]' "$LOG_PATH/bench"
  then
    echo "SUCCESS: Benchmark initialized succesfully"
  else
    echo "FAIL: Benchmark did not initialized succesfully"
    return 1
  fi
  kubectl exec -n "$CLUSTER_NAMESPACE" deploy/pgbouncer -c pgbench -- pgbench postgresql://postgres:sup3rus3r@localhost/postgres -T 120 -c 4 -j 4 -P 2 --progress-timestamp 2>&1 | tee -a "$LOG_PATH/bench" &
  BENCH_PID="$!"
  trap_kill "$BENCH_PID"
  CLUSTER_NAMESPACE="$CLUSTER_NAMESPACE" CLUSTER_NAME="$CLUSTER_NAME" sh -c '
    echo $$
    while true
    do
      echo
      echo "# $(date +%s --utc)"
      echo
      kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- env PATRONI_LOG_LEVEL=INFO patronictl list
      echo
      kubectl exec -n "$CLUSTER_NAMESPACE" deploy/pgbouncer -c pgbench -- psql -qpostgresql://pgbouncer_stats:pgb0unc3r@localhost/pgbouncer \
        -c "SHOW STATE" -c "SHOW DATABASES" -c "SHOW POOLS"
      echo
      sleep 2
    done' 2>&1 | tee "$LOG_PATH/monitor" &
  MONITOR_PID="$!"
  trap_kill "$MONITOR_PID"
  wait_until eval 'head -n 1 "$LOG_PATH/monitor" | kill -0 "$(cat)"'
  trap_kill "$(head -n 1 "$LOG_PATH/monitor")"
  sleep 5
  for ITERATION in 1 2 3 4 5
  do
    kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- env PATRONI_LOG_LEVEL=INFO \
      patronictl switchover --force "$CLUSTER_NAME" --primary "$CLUSTER_NAME"-0 --candidate "$CLUSTER_NAME"-1
    sleep 3
    kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- env PATRONI_LOG_LEVEL=INFO \
      patronictl restart --force "$CLUSTER_NAME" "$CLUSTER_NAME"-0
    kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- env PATRONI_LOG_LEVEL=INFO \
      patronictl restart --force "$CLUSTER_NAME" "$CLUSTER_NAME"-1
    sleep 3
    kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- env PATRONI_LOG_LEVEL=INFO \
      patronictl switchover --force "$CLUSTER_NAME" --primary "$CLUSTER_NAME"-1 --candidate "$CLUSTER_NAME"-0
    sleep 3
    kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- env PATRONI_LOG_LEVEL=INFO \
      patronictl restart --force "$CLUSTER_NAME" "$CLUSTER_NAME"-0
    kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- env PATRONI_LOG_LEVEL=INFO \
      patronictl restart --force "$CLUSTER_NAME" "$CLUSTER_NAME"-1
    sleep 3
  done
  wait "$BENCH_PID"
  if wait_until eval '! kill -0 "$BENCH_PID"' \
    && ! grep -q '^command terminated with exit code [1-9]' "$LOG_PATH/bench"
  then
    echo "SUCCESS: Benchmark terminated succesfully without any error after 10 switchovers"
    kill "$(head -n 1 "$LOG_PATH/monitor")" || true
    wait "$MONITOR_PID" || true
  else
    echo "FAIL: Benchmark do not terminated succesfully after 10 switchovers"
    kill "$BENCH_PID" || true
    kill "$(head -n 1 "$LOG_PATH/monitor")" || true
    wait "$MONITOR_PID" || true
    kubectl exec -n "$CLUSTER_NAMESPACE" deploy/pgbouncer -c pgbench \
      -- bash -c 'ps -ef | grep "[p]gbench" | tr -s " " | cut -d " " -f 2 | while read PID; do kill "$PID"; done' || true
    return 1
  fi
}

zerodowntime_major_version_upgrade_check() {
  UPGRADED_CLUSTER_NAME="$(get_sgcluster_name "$SPEC_NAME-upgraded")"
  STREAM_NAME="$(get_sgstreams_name "$SPEC_NAME-operation")"

  create_or_replace_cluster "$UPGRADED_CLUSTER_NAME" "$CLUSTER_NAMESPACE" "2" \
    --set-string 'cluster.postgres.version='"$E2E_MAJOR_TARGET_POSTGRES_VERSION" \
    --set instanceProfiles=null \
    --set configurations.poolingconfig.create=false \
    --set-string cluster.configurations.sgPostgresConfig=postgresconf-upgraded \
    --set-string 'cluster.configurations.credentials.users.superuser.password.name=credentials' \
    --set-string 'cluster.configurations.credentials.users.superuser.password.key=superuser-password' \
    --set-string 'cluster.configurations.patroni.initialConfig.postgresql.callbacks.on_role_change=/callbacks/pre_promote' \
    --set-string 'cluster.configurations.patroni.initialConfig.postgresql.before_stop=/callbacks/before_stop' \
    --set-string 'cluster.pods.customVolumeMounts.patroni[0].name=custom-callbacks' \
    --set-string 'cluster.pods.customVolumeMounts.patroni[0].mountPath=/callbacks' \
    --set-string 'cluster.pods.customVolumes[0].name=callbacks' \
    --set-string 'cluster.pods.customVolumes[0].configMap.name=callbacks' \
    --set 'cluster.pods.customVolumes[0].configMap.defaultMode='"$(printf %d 0555)" \
    --set 'cluster.pods.customVolumes[0].configMap.optional=false'
  wait_cluster "$UPGRADED_CLUSTER_NAME" "$CLUSTER_NAMESPACE"
  kubectl exec -n "$CLUSTER_NAMESPACE" deploy/pgbouncer -c pgbench -- pgbench postgresql://postgres:sup3rus3r@localhost/postgres -i 2>&1 | tee "$LOG_PATH/bench"
  if ! grep -q '^command terminated with exit code [1-9]' "$LOG_PATH/bench"
  then
    echo "SUCCESS: Benchmark initialized succesfully"
  else
    echo "FAIL: Benchmark did not initialized succesfully"
    return 1
  fi

  cat << EOF | kubectl exec -i -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- psql -v ON_ERROR_STOP=1 2>&1 | tee "$LOG_PATH/bench"
ALTER TABLE pgbench_history ADD COLUMN id serial;
UPDATE pgbench_history SET id = nextval('pgbench_history_id_seq');
ALTER TABLE pgbench_history ADD PRIMARY KEY(id);
EOF
  if ! grep -q '^command terminated with exit code [1-9]' "$LOG_PATH/bench"
  then
    echo "SUCCESS: Benchmark initialized succesfully"
  else
    echo "FAIL: Benchmark did not initialized succesfully"
    return 1
  fi

  kubectl exec -n "$CLUSTER_NAMESPACE" deploy/pgbouncer -c pgbench -- pgbench postgresql://postgres:sup3rus3r@localhost/postgres -T 180 -c 4 -j 4 -P 2 --progress-timestamp 2>&1 | tee -a "$LOG_PATH/bench" &
  BENCH_PID="$!"
  trap_kill "$BENCH_PID"
  CLUSTER_NAMESPACE="$CLUSTER_NAMESPACE" CLUSTER_NAME="$CLUSTER_NAME" UPGRADED_CLUSTER_NAME="$UPGRADED_CLUSTER_NAME" sh -c '
    echo $$
    while true
    do
      date +%s --utc
      echo
      kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- env PATRONI_LOG_LEVEL=INFO patronictl list
      echo
      kubectl exec -n "$CLUSTER_NAMESPACE" "$UPGRADED_CLUSTER_NAME"-0 -c patroni -- env PATRONI_LOG_LEVEL=INFO patronictl list
      echo
      kubectl exec -n "$CLUSTER_NAMESPACE" deploy/pgbouncer -c pgbench -- psql postgresql://pgbouncer_stats:pgb0unc3r@localhost/pgbouncer \
        -c "SHOW STATE" -c "SHOW DATABASES" -c "SHOW POOLS" -c "SHOW SERVERS"
      echo
      sleep 2
    done' 2>&1 | tee "$LOG_PATH/monitor" &
  MONITOR_PID="$!"
  trap_kill "$MONITOR_PID"
  wait_until eval 'head -n 1 "$LOG_PATH/monitor" | kill -0 "$(cat)"'
  trap_kill "$(head -n 1 "$LOG_PATH/monitor")"
  sleep 5

  cat << EOF | tee "$LOG_PATH/sgstream.yaml" | kubectl replace --force -f -
apiVersion: stackgres.io/v1alpha1
kind: SGStream
metadata:
  namespace: $CLUSTER_NAMESPACE 
  name: "$STREAM_NAME"
spec:
  maxRetries: 0
  source:
    type: SGCluster
    sgCluster:
      name: "$CLUSTER_NAME"
      excludes:
      - cron\..*
  target:
    type: SGCluster
    sgCluster:
      name: "$UPGRADED_CLUSTER_NAME"
  pods:
    persistentVolume:
      size: 1Gi
EOF

  if wait_until eval 'kubectl get sgstream -n "$CLUSTER_NAMESPACE" "$STREAM_NAME" -o json | jq .status.snapshot.snapshotCompleted | grep -qxF true'
  then
    echo "SUCCESS: snapshot completed"
  else
    echo "FAIL: snapshot did not complete"
    return 1
  fi

  cat << 'EOF' | kubectl exec -i -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- psql -v ON_ERROR_STOP=1
CREATE TABLE heartbit (id serial, time timestamp DEFAULT clock_timestamp(), PRIMARY KEY (id));
INSERT INTO heartbit DEFAULT VALUES;
EOF

  if wait_until eval '[ "x$(kubectl exec -i -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- psql -tA -c "SELECT extract(EPOCH from MAX(time))::bigint FROM heartbit")" \
    = "x$(kubectl exec -i -n "$CLUSTER_NAMESPACE" "$UPGRADED_CLUSTER_NAME"-0 -c patroni -- psql -tA -c "SELECT extract(EPOCH from MAX(time))::bigint FROM heartbit")" ]'
  then
    echo "SUCCESS: first heartbit in sync"
  else
    echo "FAIL: first heartbit not in sync"
    return 1
  fi

  echo "Switching from $CLUSTER_NAME SGCluster ($E2E_MAJOR_SOURCE_POSTGRES_VERSION) to $UPGRADED_CLUSTER_NAME SGCluster ($E2E_MAJOR_TARGET_POSTGRES_VERSION)..."
  kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- patronictl pause
  kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- psql postgresql://pgbouncer_admin:pgb0unc3r@pgbouncer/pgbouncer -c PAUSE
  kubectl patch -n "$CLUSTER_NAMESPACE" service entrypoint --type merge -p "
  spec:
    externalName: "$UPGRADED_CLUSTER_NAME.$CLUSTER_NAMESPACE.svc.cluster.local"
  " -o yaml
  cat << 'EOF' | kubectl exec -i -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- psql -v ON_ERROR_STOP=1
INSERT INTO heartbit DEFAULT VALUES;
EOF


  if wait_until eval '[ "x$(kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME"-0 -c patroni -- psql -tA -c "SELECT extract(EPOCH from MAX(time))::bigint FROM heartbit")" \
    = "x$(kubectl exec -n "$CLUSTER_NAMESPACE" "$UPGRADED_CLUSTER_NAME"-0 -c patroni -- psql -tA -c "SELECT extract(EPOCH from MAX(time))::bigint FROM heartbit")" ]'
  then
    echo "SUCCESS: final heartbit in sync"
  else
    echo "FAIL: final heartbit not in sync"
    return 1
  fi

  kubectl annotate sgstream -n "$CLUSTER_NAMESPACE" "$STREAM_NAME" debezium-signal.stackgres.io/tombstone=
  if kubectl wait --timeout="${E2E_TIMEOUT}s" sgstream -n "$CLUSTER_NAMESPACE" "$STREAM_NAME" --for=condition=Completed
  then
    echo "SUCCESS: stream has completed"
  else
    echo "FAIL: stream has not completed"
    return 1
  fi

  cat << 'EOF' | kubectl exec -i -n "$CLUSTER_NAMESPACE" "$UPGRADED_CLUSTER_NAME"-0 -c patroni -- psql -v ON_ERROR_STOP=1
SELECT setval('pgbench_history_id_seq'::regclass, (SELECT MAX(id) + 1 FROM pgbench_history));
DROP TABLE heartbit;
EOF

  kubectl exec -n "$CLUSTER_NAMESPACE" "$UPGRADED_CLUSTER_NAME"-0 -c patroni -- psql postgresql://pgbouncer_admin:pgb0unc3r@pgbouncer/pgbouncer -c RESUME
  echo "...switch to $UPGRADED_CLUSTER_NAME SGCluster ($E2E_MAJOR_TARGET_POSTGRES_VERSION) completed!"

  wait "$BENCH_PID"
  if wait_until eval '! kill -0 "$BENCH_PID"' \
    && ! grep -q '^command terminated with exit code [1-9]' "$LOG_PATH/bench"
  then
    echo "SUCCESS: Benchmark terminated succesfully without any error after 10 switchovers"
    kill "$(head -n 1 "$LOG_PATH/monitor")" || true
    wait "$MONITOR_PID" || true
  else
    echo "FAIL: Benchmark do not terminated succesfully after 10 switchovers"
    kill "$BENCH_PID" || true
    kill "$(head -n 1 "$LOG_PATH/monitor")" || true
    wait "$MONITOR_PID" || true
    kubectl exec -n "$CLUSTER_NAMESPACE" deploy/pgbouncer -c pgbench \
      -- bash -c 'ps -ef | grep "[p]gbench" | tr -s " " | cut -d " " -f 2 | while read PID; do kill "$PID"; done' || true
    return 1
  fi
}
